import requests
import re
from fake_useragent import UserAgent
import os
import sqlite3

class douban ():
    #随机访问头，并获取网页数据
    def getdata(self,url):
        ua = UserAgent()
        headers = {'User-Agent': ua.random}
        # time.sleep(random.random() * 3)   #增加成功率，但会影响速度
        try:
            r = requests.get(url, timeout=30, headers=headers)
            r.encoding = 'utf-8'
            r.raise_for_status()
        except:
            r = requests.get(url, timeout=30, headers=headers)
        return r.text
    #主程序
    def run(self,a):
        global data_list
        global data_lists
        c = str(a)
        url = 'https://movie.douban.com/top250?start='+c+'&filter='  #网址
        data = self.getdata(url)
        name_list = self.re_list(data,''' <a href=".*?" class="">
                            <span class="title">(.*?)</span>''')    #电影名
        lianjie_list = self.re_list(data,'''</em>
                    <a href="(.*?)">''') #链接
        pingfen_list = self.re_list(data,'''<span class="rating_num" property="v:average">(.*?)</span>''')  #评分
        pingjia_list = self.re_list(data,'''<span property="v:best" content="10.0"></span>
                                <span>(.*?)人评价</span>''')   #评价数
        jianjie_list_data = self.re_list(data,'''<span>.*?人评价</span>(.*?)</li>''')  #电影概括
        jianjie_list = []
        for n in jianjie_list_data:
            jianjie= self.re_list(n,'''<span class="inq">(.*?)</span>''')
            if len(jianjie) ==0:
                jianjie_list.append('')
            else:
                jianjie_list.append(jianjie[0])
        tupian_list = self.re_list(data,'<img width="100" alt=".*?" src="(.*?)" class="">')
        other_name = self.re_list(data,'<span class="other">&nbsp;/&nbsp;(.*?)</span>')
        xinxi_list0 = self.re_list(data,'''<div class="bd">
                        <p class="">
                            (.*?)
                        </p>''')  #导演演员
        xinxi_list = []
        for n in xinxi_list0:
            n=n.replace('&nbsp','').replace('<br>','').replace(';','').replace('/','').replace("'",'')\
                .replace(' ','').replace('n','')
            n = re.sub(r'\n','',n)
            xinxi_list.append(n)
        print(xinxi_list)
        #创建文件夹
        # if not os.path.exists('D://moviep/'):
        #     os.mkdir('D://moviep/')
        # temp = 0
        # global f
        # #储存电影图片
        # for n in tupian_list:
        #     path = 'D://moviep/'+name_list[temp]+'.jpg'
        #     temp = temp+1
        #     picture_123 = requests.get(n)
        #     if not os.path.exists(path):
        #         with open(path,'wb')as f:
        #              f.write(picture_123.content)
        #     else:print('文件已存在')
        for n in range(25):
            data_list = [name_list[n],other_name[n],xinxi_list[n],lianjie_list[n],tupian_list[n],pingfen_list[n],pingjia_list[n],jianjie_list[n]]
            data_lists.append(data_list)
            print(len(data_lists))



    def savedata_first(self):
        #创建数据库
        c = sqlite3.connect('movie.db')
        print('成功打开数据库')
        #建表
        cur = c.cursor()  # 获取游标
        sql = '''
            create table movietop250
                (id integer primary key autoincrement,
                name varchar,
                other_name char(50),
                xinxi varchar ,
                movie_lianjie text,
                tupian_lianjie text,
                pingfen float,
                pingjia_num int,
                duanping char(50)
                );

        '''
        cur.execute(sql)  # 执行sql语句
        c.commit()  # 提交数据库操作
        c.close()  # 关闭数据库连接
        print('成功建表')

    #将数据储存在数据库
    def savedata(self):
        try:
            self.savedata_first()
        except:
            print('数据库已存在')
        c = sqlite3.connect('movie.db')
        cur = c.cursor()
        for data in data_lists:
            for index in range(len(data)):
                if index == 5 or index == 6 :
                    continue
                data[index] = '"'+data[index]+'"'
            sql = '''insert into movietop250
                    (name , other_name,xinxi,movie_lianjie, tupian_lianjie, pingfen,pingjia_num,duanping)
                        values (%s)''' %','.join(data)
            print(sql)
            cur.execute(sql)
            c.commit()
        cur.close()
        c.close()

    def re_list(self,data,a):
        r = re.findall(a,data,re.S)
        return r



if __name__ == '__main__':
    data_list = []
    data_lists = []
    top250 = douban()
    for n in range(0,10):
        a = n*25
        print('爬取进度%d/250' % a)
        top250.run(a)
    print(data_lists)
    top250.savedata()
    # f.close()
